// PageProcessor.h
#pragma once
#include <string>
#include <vector>
#include <set>
#include "cppjieba/Jieba.hpp"
#include "simhash/Simhasher.hpp"
#include "TextUtils.h"
#include "DirectoryScanner.h"
#include "XMLUtils.h"
#include <map>
#include <bitset>
#include "simhash/Simhasher.hpp" 
#include <fstream>

using std::string;
using std::set;
using std::vector;
using std::map;
using std::set;
using std::ostream;
using std::to_string;
using namespace simhash;

class PageProcessor
{
public:
    PageProcessor();
    void process(const string& dir);

private:
    void extract_documents(const string& dir);
    void deduplicate_documents();
    void build_pages_and_offsets(const string& pages, const string& offsets);
    void build_inverted_index(const string& filename);
    void clean_content_text(string& content);
private:
    struct Document{
        int id;
        string link;
        string title;
        string content;
    };

    // 友元声明
    friend void add_document_by_xml(const string& dir, vector<PageProcessor::Document>& document);
    
private:
    cppjieba::Jieba m_tokenizer;
    simhash::Simhasher m_hasher;
    set<string> m_stopWords;    // 使用set, 而非vector, 是为了方便查找
    vector<Document> m_documents;
    map<string, map<int, double>> m_invertedIndex;
};

